iqbal-poetry-translation:
  id: iqbal-poetry-translation.dev.v0
  description: Test the model's ability to correctly translate Iqbal's poetry into English. Translating poetry is a tricky task, and almost every translation tool fails at this.
  metrics: [accuracy]
iqbal-poetry-translation.dev.v0:
  class: evals.elsuite.modelgraded.classify:ModelBasedClassify
  args:
    samples_jsonl: iqbal-poetry-translation/samples.jsonl
    eval_type: cot_classify
    modelgraded_spec: translation
    modelgraded_spec_args:
      language: "Urdu"

# a meta-evaluation of the above modelgraded eval
# this example uses a labeled dataset with "completion" and "choice"
translation-meta:
  id: translation-meta.dev.v0
  metrics: [accuracy]
translation-meta.dev.v0:
  class: evals.elsuite.modelgraded.classify:ModelBasedClassify
  args:
    samples_jsonl: iqbal-poetry-translation/labeled-samples.jsonl
    eval_type: cot_classify
    modelgraded_spec: translation
    metaeval: true
    modelgraded_spec_args:
      language: "Urdu"
